package com.xiaotu.spider.project.pageprocessor;

import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Component;

import com.xiaotu.common.util.DateUtils;
import com.xiaotu.common.util.FileUtil;
import com.xiaotu.common.util.HtmlUtil;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.RegexUtils;
import com.xiaotu.common.util.SepratorUtil;
import com.xiaotu.spider.AUrlPageProcessor;

import us.codecraft.webmagic.Page;

/**
 * @类名 CBJPageProcessor
 * @日期 2017年5月22日
 * @作者 高海军
 * @功能 中国经营报文章内容爬取
 */
@Component("CBJPageProcessor")
public class CBJPageProcessor extends AUrlPageProcessor
{
	private static final String CSV_SPLITOR = "#***#";
	
	@Override
	public void process(Page page)
	{
		String url = page.getUrl().toString();
		if (url.endsWith("period.xml"))
			addPageLink(page);
		else if (RegexUtils.regexMatch(
				".*(node_" + RegexUtils.REGEX_INTEGER + ".htm)$", url))
			this.addVersionAndArticleLink(page);
		else
			this.articleProcess(page);
	}
	
	private void addVersionAndArticleLink(Page page)
	{
		if (page.getUrl().toString().endsWith("node_1.htm"))
		{
			List<String> linkList = page.getHtml()
					.xpath("//td[@class='default']/a[@id='pageLink']/@href")
					.all();
			PageUtils.addTargetRequests(page,
					linkList.subList(1, linkList.size()),
					page.getRequest().getExtras());
		}
		
		PageUtils.addTargetRequests(page,
				page.getHtml().xpath("//div[@id='btdh']//a/@href").all(),
				page.getRequest().getExtras());
	}
	
	private void addPageLink(Page page)
	{
		List<String> dateList = RegexUtils.regexSearchAll(
				"[0-9]{4}-[0-9]{2}-[0-9]{2}", page.getRawText());
		Set<String> dateSet = new HashSet<>(dateList);
		
		for (String dateStr : dateSet)
		{
			String[] dateArr = dateStr.split(SepratorUtil.SEP_CROSS_EN);
			PageUtils.addTargetRequest(page,
					"http://dianzibao.cb.com.cn/html/" + dateArr[0]
							+ SepratorUtil.SEP_CROSS_EN + dateArr[1] + "/"
							+ dateArr[2] + "/node_1.htm",
					page.getRequest().getExtras());
		}
	}
	
	private void articleProcess(Page page)
	{
		String content = page.getHtml().xpath("//div[@id='ozoom']").get();
		if (StringUtils.isEmpty(content))
			return;
		content = HtmlUtil.delHTMLTag(content);
		if (StringUtils.isEmpty(content))
			return;
		content = content.replaceAll("&nbsp;", StringUtils.EMPTY)
				.replaceAll("SourcePh style=display:none>", StringUtils.EMPTY);
		if (StringUtils.isEmpty(content))
			return;
		String title = page.getHtml().xpath("//td[@class='font01']/text()")
				.get();
		String date = page.getHtml()
				.xpath("//span[@class='default']/strong/text()")
				.regex("[0-9]{4}年[0-9]{2}月[0-9]{2}日").get();
		
		FileUtil.writeFile("D://article_2017.csv",
				page.getUrl().toString() + CSV_SPLITOR + date + CSV_SPLITOR
						+ title + CSV_SPLITOR + content
						+ SepratorUtil.LINE_SEPRATOR);
	}
	
	@Override
	protected String[] getStartRequestUrls()
	{
		Date startDate = DateUtils.getDate("2017-04", DateUtils.MONTH_FORMAT);
		// Date endDate = DateUtils.getDate("2017-06", DateUtils.MONTH_FORMAT);
		Date endDate = DateUtils.getDate("2017-06", DateUtils.MONTH_FORMAT);
		Date currDate = startDate;
		
		List<String> linkList = new ArrayList<String>();
		do
		{
			linkList.add("http://dianzibao.cb.com.cn/html/"
					+ DateUtils.getDateString(currDate, DateUtils.MONTH_FORMAT)
					+ "/period.xml");
			currDate = DateUtils.getNextMonth(currDate);
		}
		while (currDate.before(endDate));
		
		String[] urlArr = new String[linkList.size()];
		linkList.toArray(urlArr);
		return urlArr;
	}
	
	@Override
	protected String getSpiderType()
	{
		return "CBJ";
	}
	
	@Override
	public String getPageDataType(Page page)
	{
		return "CBJ";
	}
	
}
